import lxml.html
import requests
from lxml import etree


def spider_index(school, school_url, academy, academy_url):
    s = requests.session()
    base_url = "https://jyxy.jxnu.edu.cn"
    t = s.get("https://jyxy.jxnu.edu.cn/1497/list.htm").text
    t = t.encode("ISO-8859-1").decode("utf-8")
    # print(t)
    etr = etree.HTML(t)
    li = etr.xpath("//div[@class='col_list']/ul/li")

    for i in li:
        h = etree.tostring(i)
        s_etr = etree.HTML(h.decode())
        s = s_etr.xpath("//a/@href")[0]
        s1 = s_etr.xpath("//a/span/text()")[0]
        list_url = base_url + s
        get_next(list_url, s1, school, school_url, academy, academy_url)


def get_next(list_url, s1, school, school_url, academy, academy_url):
    print(list_url)
    s = requests.session()
    t = s.get(list_url).text
    t = t.encode("ISO-8859-1").decode("utf-8")
    # print(t)
    etr = etree.HTML(t)
    li = etr.xpath("//div[@id='wp_content_w6_0']/p")

    etr1 = etree.HTML(t)
    li1 = etr1.xpath("//div[@id='wp_content_w6_0']/table")

    ch = []
    for l1 in li:
        s = str(l1.xpath("string()")).encode().replace(b"\xc2\xa0", b"")
        if len(s) > 10:
            ch.append(s.decode())
    # print(li1)
    print(ch)
    zi = zip(ch, li1)
    for professional, i in zi:
        dom_text = lxml.html.tostring(i)
        etr2 = etree.HTML(dom_text)
        td = etr2.xpath("//td")
        # print(td)
        for ttd in td:
            try:
                dom_text_td = lxml.html.tostring(ttd)
                etr3 = etree.HTML(dom_text_td)
                mainurl = etr3.xpath("//a/@href")[0]
                t1 = etr3.xpath("string()")
                # print(list_url, s1, school, school_url, academy, academy_url, professional, mainurl, t1)
            except IndexError:
                pass


if __name__ == '__main__':
    spider_index("江西师范大学", "https://www.jxnu.edu.cn/", "教育学院", "http://jyxy.jxnu.edu.cn/")
